/**
 * This class was created by cry. It's distributed as
 * part of the myspidertest Mod.
 *
 * 版权所有(C) 上海纯米电子科技有限公司 2014-2023
 * Copyright 2014-2023 CHUNMI TECHNOLOGY CO..
 *
 * This software is the confidential and proprietary information of
 * CHUNMI Corporation ("Confidential Information").  You
 * shall not disclose such Confidential Information and shall use
 * it only in accordance with the terms of the license agreement
 * you entered into with CHUNMI.
 *
 * File Created @ [Feb 9, 2017, 10:49:02 AM (CST)]
 */
package spider.processor;

import java.util.List;

import javax.management.JMException;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class OschinaBlogPageProcesser implements PageProcessor {

	private Site site = Site.me().setDomain("my.oschina.net");

    @Override
    public void process(Page page) {
        List<String> links = page.getHtml().links().regex("https://my\\.oschina\\.net/flashsword/blog/\\d+").all();
        System.out.println("&&&&&&&&&&&    "+links);
        page.addTargetRequests(links);
//        page.putField("title", page.getHtml().xpath("//div[@class='BlogEntity']/div[@class='BlogTitle']/h1").toString());
//        page.putField("content", page.getHtml().$("div.content").toString());
//        page.putField("tags",page.getHtml().xpath("//div[@class='BlogTags']/a/text()").all());
        page.putField("title", page.getHtml().xpath("//a[@class='blog-title']/text()").all());
    }

    @Override
    public Site getSite() {
        return site;

    }

    public static void main(String[] args) throws JMException {
        Spider.create(new OschinaBlogPageProcesser()).addUrl("https://my.oschina.net/flashsword/blog")
             .addPipeline(new ConsolePipeline()).run();
    	
//    	Spider oschinaSpider = Spider.create(new OschinaBlogPageProcesser())
//                .addUrl("http://my.oschina.net/flashsword/blog");
//
//        SpiderMonitor.instance().register(oschinaSpider);
//        oschinaSpider.start();
    }

}
